spb/social-runtime-crawler
Public
TypeScript 91.8%
HTML 3.2%
JavaScript 3%
SQL 1.4%
CSS 0.7%
1"use client";23import { use, useMemo, useState } from "react";4import Link from "next/link";5import clsx from "clsx";6import { ExternalLink } from "lucide-react";7import { useApi } from "@/lib/api";8import { ago, eventColor, fmtDate, fmtDuration, fmtTime, truncate } from "@/lib/format";9import { Bar, Empty, Metric, Panel, Skeleton, StatusPill, Tag, Table } from "@/components/ui";10import { GainChart } from "@/components/charts";11import { LiveFeed } from "@/components/LiveFeed";12import { WorldGraph, type GraphEdge, type GraphNode } from "@/components/WorldGraph";13import { EntityTable, type EntityRow } from "@/components/EntityTable";1415interface Session {16 session_id: string;17 platform: string;18 account_alias: string;19 mode: string | null;20 goal: string | null;21 started_at: string;22 ended_at: string | null;23 health: string;24 job: { budget?: Record<string, number>; query?: string; media_level?: number } | null;25 summary: { steps: number; entities: number; videos: number; network_responses: number; schemas: number; patterns_learned: number; ended_because: string; world: { nodes: number; edges: number; visited: number; multi_surface: number; by_type: Record<string, number> }; connector: { confidence: number; page_types: number; network_schemas: number } } | null;26 event_counts: { event_type: string; n: number }[];27}28interface Action {29 step: number;30 action_type: string;31 label: string;32 target_url: string | null;33 planner: string;34 expected_gain: number;35 novelty: number;36 relevance: number;37 reason: string;38 scores: { action_id: string; information_gain: number; penalties: string[] }[];39 before_state: { url: string; page_type: string; visible_entities: number } | null;40 after_state: { url: string; page_type: string; visible_entities: number; new_entities: number; network_responses: number; dom_nodes_added: number } | null;41 success: boolean;42 error: string | null;43 duration_ms: number;44 ts: string;45}46interface PageRow {47 step: number;48 ts: string;49 payload: { url: string; title: string; page_type: string; confidence: number; signals: string[]; entities: number; dom_entities: number; network_entities: number; both_surfaces: number; field_agreements: number; field_conflicts: { field: string; network: unknown; dom: unknown }[]; videos: number; summary: string };50}51interface Media {52 fingerprint: string;53 media_type: string;54 platform_media_id: string | null;55 title: string | null;56 author: string | null;57 duration_s: number | null;58 width: number | null;59 height: number | null;60 thumbnail_url: string | null;61 page_url: string | null;62 delivery: { kind: string; hostnames: string[]; manifest_url?: string } | null;63 frames: string[] | null;64}65interface Schema {66 shape_hash: string;67 fingerprint: { hostname: string; path_pattern: string; method: string; graphql_operation?: string };68 schema: { candidate_entity_types: { type: string; confidence: number; path: string }[]; fields: { path: string; semantic?: { kind: string; confidence: number }; example?: string }[]; object_count: number };69 observed_count: number;70 sample_url: string;71}7273const TABS = ["decisions", "pages", "entities", "runtime apis", "media", "world model", "events"] as const;7475export default function SessionConsole({ params }: { params: Promise<{ id: string }> }) {76 const { id } = use(params);77 const [tab, setTab] = useState<(typeof TABS)[number]>("decisions");78 const s = useApi<Session>(`/sessions/${id}`, { refreshMs: 10_000 });79 const live = !s.data?.ended_at;80 const actions = useApi<Action[]>(`/sessions/${id}/actions`, { refreshMs: live ? 5000 : undefined });81 const pages = useApi<PageRow[]>(`/sessions/${id}/pages`, { refreshMs: live ? 5000 : undefined });82 const entities = useApi<EntityRow[]>(tab === "entities" ? `/sessions/${id}/entities?limit=500` : null, { refreshMs: live ? 8000 : undefined });83 const media = useApi<Media[]>(tab === "media" ? `/sessions/${id}/media` : null);84 const schemas = useApi<Schema[]>(tab === "runtime apis" ? `/sessions/${id}/schemas` : null);85 const world = useApi<{ nodes: GraphNode[]; edges: GraphEdge[] }>(tab === "world model" ? `/sessions/${id}/world` : null, { refreshMs: live ? 10_000 : undefined });86 const [selStep, setSelStep] = useState<number | null>(null);8788 const gainSeries = useMemo(() => (actions.data ?? []).map((a) => ({ step: a.step, expected_gain: Number(a.expected_gain), novelty: Number(a.novelty), relevance: Number(a.relevance), action_type: a.action_type, new_entities: a.after_state?.new_entities })), [actions.data]);89 const sess = s.data;90 const sum = sess?.summary;91 const lastPage = pages.data?.[pages.data.length - 1];92 const selPage = selStep !== null ? pages.data?.find((p) => p.step === selStep) : lastPage;9394 return (95 <div className="space-y-4">96 <div className="flex flex-wrap items-start gap-3">97 <div className="min-w-0">98 <div className="flex items-center gap-2 text-xs text-dim mono">99 <Link href="/sessions" className="hover:text-cyan">sessions</Link> / {id}100 </div>101 <h1 className="text-xl font-semibold tracking-tight mt-1 truncate max-w-3xl">{sess?.goal ?? "…"}</h1>102 <div className="flex flex-wrap items-center gap-2 mt-2 text-xs">103 {sess && <StatusPill status={sess.ended_at ? (sess.health === "auth_required" ? "auth_required" : "done") : sess.health} />}104 {sess && <Tag>{sess.platform}</Tag>}105 {sess?.mode && <Tag color="#ffb347">{sess.mode}</Tag>}106 {sess?.job?.query && <Tag color="#9d7bff">“{sess.job.query}”</Tag>}107 {sess && <span className="text-dim">started {fmtDate(sess.started_at)} · {sess.ended_at ? `ended ${ago(sess.ended_at)}` : "running"}</span>}108 {sum && <span className="text-dim">· ended because <span className="text-fg-2">{sum.ended_because}</span></span>}109 </div>110 </div>111 <div className="ml-auto grid grid-cols-3 sm:grid-cols-6 gap-2 text-center">112 {[113 ["steps", actions.data?.length ?? sum?.steps],114 ["entities", sum?.entities ?? sess?.event_counts.filter((c) => /DISCOVERED/.test(c.event_type)).reduce((a, c) => a + c.n, 0)],115 ["videos", sum?.videos],116 ["responses", sum?.network_responses ?? sess?.event_counts.find((c) => c.event_type === "NETWORK_RESPONSE_OBSERVED")?.n],117 ["schemas", sum?.schemas ?? sess?.event_counts.find((c) => c.event_type === "NETWORK_SCHEMA_DISCOVERED")?.n],118 ["learned", sum?.patterns_learned ?? sess?.event_counts.find((c) => c.event_type === "CONNECTOR_PATTERN_LEARNED")?.n],119 ].map(([k, v]) => (120 <div key={k as string} className="panel px-3 py-2">121 <div className="mono text-lg text-cyan">{v ?? "—"}</div>122 <div className="text-[10px] uppercase tracking-widest text-dim">{k as string}</div>123 </div>124 ))}125 </div>126 </div>127128 <div className="flex gap-1 border-b border-line text-sm">129 {TABS.map((t) => (130 <button key={t} onClick={() => setTab(t)} className={clsx("px-3 py-2 -mb-px border-b-2 capitalize transition", tab === t ? "border-cyan text-cyan" : "border-transparent text-fg-2 hover:text-fg")}>131 {t}132 </button>133 ))}134 </div>135136 {tab === "decisions" && (137 <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">138 <div className="xl:col-span-2 space-y-4">139 <Panel title="Information gain per step" sub="what the agent expected of each move">140 {gainSeries.length ? <GainChart data={gainSeries} /> : <Empty>no decisions yet</Empty>}141 </Panel>142 <Panel title="Agent decisions" sub="why did the crawler do that? — click a step to see its page state" pad={false}>143 {actions.loading ? (144 <div className="p-4">145 <Skeleton rows={6} />146 </div>147 ) : (148 <ul className="divide-y divide-line/60 max-h-[720px] overflow-auto scrollbar-thin">149 {(actions.data ?? [])150 .slice()151 .reverse()152 .map((a) => (153 <li key={a.step} onClick={() => setSelStep(a.step)} className={clsx("px-4 py-2.5 cursor-pointer hover:bg-white/[0.03]", selStep === a.step && "bg-cyan/5")}>154 <div className="flex items-center gap-2 text-sm">155 <span className="mono text-dim w-8">#{a.step}</span>156 <Tag color={a.success ? "#43e69a" : "#ff5c7a"}>{a.action_type.replace(/_/g, " ").toLowerCase()}</Tag>157 <span className="truncate text-fg">{truncate(a.label.replace(/^Open \w+ E\d+: /, ""), 80)}</span>158 <span className="ml-auto mono text-xs text-amber whitespace-nowrap">gain {Number(a.expected_gain).toFixed(3)}</span>159 </div>160 <div className="flex flex-wrap gap-x-4 text-[11px] text-dim mono mt-1 ml-10">161 <span>nov {Number(a.novelty).toFixed(2)}</span>162 <span>rel {Number(a.relevance).toFixed(2)}</span>163 <span>{a.planner}</span>164 {a.after_state && (165 <span>166 → {a.after_state.page_type} · +{a.after_state.new_entities} ent · {a.after_state.network_responses} resp · {a.duration_ms} ms167 </span>168 )}169 {a.error && <span className="text-red">{truncate(a.error, 60)}</span>}170 </div>171 <div className="text-xs text-fg-2 mt-1 ml-10">{a.reason}</div>172 </li>173 ))}174 </ul>175 )}176 </Panel>177 </div>178 <div className="space-y-4">179 <Panel title={selStep !== null ? `Page state after step ${selStep}` : "Current page state"} sub="semantic DOM representation given to the planner">180 {selPage ? (181 <div className="space-y-2">182 <div className="flex items-center gap-2 text-xs">183 <Tag color="#38e1ff">{selPage.payload.page_type}</Tag>184 <span className="mono text-dim">conf {Number(selPage.payload.confidence).toFixed(2)}</span>185 <a href={selPage.payload.url} target="_blank" rel="noreferrer" className="ml-auto text-dim hover:text-cyan">186 <ExternalLink className="h-3.5 w-3.5" />187 </a>188 </div>189 <div className="text-[11px] text-dim break-all">{selPage.payload.url}</div>190 <div className="grid grid-cols-4 gap-2 text-center text-[10.5px] mono">191 {[192 ["entities", selPage.payload.entities],193 ["dom", selPage.payload.dom_entities],194 ["network", selPage.payload.network_entities],195 ["both", selPage.payload.both_surfaces],196 ].map(([k, v]) => (197 <div key={k as string} className="rounded bg-bg-2 py-1">198 <div className="text-fg">{v as number}</div>199 <div className="text-dim">{k as string}</div>200 </div>201 ))}202 </div>203 {selPage.payload.field_conflicts?.length > 0 && (204 <div className="text-[11px] text-amber">205 {selPage.payload.field_conflicts.length} field conflict(s): {selPage.payload.field_conflicts.map((c) => c.field).join(", ")}206 </div>207 )}208 <pre className="mono text-[11px] leading-5 text-fg-2 whitespace-pre-wrap max-h-[520px] overflow-auto scrollbar-thin bg-bg-2 rounded p-3">{selPage.payload.summary}</pre>209 </div>210 ) : (211 <Empty>no page state yet</Empty>212 )}213 </Panel>214 <Panel title="Budget">215 {sess?.job?.budget ? (216 <div>217 <Metric label="max minutes" value={sess.job.budget.max_minutes} />218 <Metric label="max actions" value={sess.job.budget.max_actions} />219 <Metric label="max profiles opened" value={sess.job.budget.max_profiles} />220 <Metric label="max videos opened" value={sess.job.budget.max_videos} />221 <Metric label="media level" value={sess.job.media_level ?? 1} />222 <div className="mt-3 text-[10.5px] text-dim">steps used</div>223 <Bar value={(actions.data?.length ?? 0) / Math.max(1, sess.job.budget.max_actions)} color="#ffb347" />224 </div>225 ) : (226 <Skeleton rows={3} />227 )}228 </Panel>229 </div>230 </div>231 )}232233 {tab === "pages" && (234 <Panel title="Navigation path" sub="each page the browser reached, classified" pad={false}>235 <div className="p-4">236 {pages.data?.length ? (237 <Table head={["step", "time", "page type", "conf", "url", "entities (dom / net / both)", "agreements", "videos"]} dense>238 {pages.data.map((p) => (239 <tr key={`${p.step}-${p.ts}`} onClick={() => setSelStep(p.step)} className="cursor-pointer">240 <td className="mono">#{p.step}</td>241 <td className="mono text-dim">{fmtTime(p.ts)}</td>242 <td>243 <Tag color="#38e1ff">{p.payload.page_type}</Tag>244 </td>245 <td className="mono">{Number(p.payload.confidence).toFixed(2)}</td>246 <td className="max-w-[420px] truncate text-fg-2">247 <a href={p.payload.url} target="_blank" rel="noreferrer" className="hover:text-cyan">248 {truncate(p.payload.url, 80)}249 </a>250 </td>251 <td className="mono">252 {p.payload.entities} <span className="text-dim">({p.payload.dom_entities} / {p.payload.network_entities} / {p.payload.both_surfaces})</span>253 </td>254 <td className="mono">255 {p.payload.field_agreements}256 {p.payload.field_conflicts?.length ? <span className="text-amber"> / {p.payload.field_conflicts.length}✗</span> : ""}257 </td>258 <td className="mono">{p.payload.videos}</td>259 </tr>260 ))}261 </Table>262 ) : (263 <Empty>no pages yet</Empty>264 )}265 </div>266 </Panel>267 )}268269 {tab === "entities" && (270 <Panel title="Entities observed in this session" sub="every field keeps its provenance (network / dom) and confidence" pad={false}>271 <div className="p-4">{entities.loading ? <Skeleton rows={8} /> : entities.data?.length ? <EntityTable rows={entities.data} /> : <Empty>no entities yet</Empty>}</div>272 </Panel>273 )}274275 {tab === "runtime apis" && (276 <Panel title="Runtime API discovery" sub="response shapes fingerprinted on this platform — no endpoint was hardcoded" pad={false}>277 <div className="p-4 space-y-3">278 {schemas.loading ? (279 <Skeleton rows={6} />280 ) : (281 (schemas.data ?? []).map((sc) => (282 <details key={sc.shape_hash} className="rounded-lg border border-line bg-bg-2/40 open:bg-bg-2/70">283 <summary className="cursor-pointer px-3 py-2 flex flex-wrap items-center gap-2 text-sm">284 <Tag color="#9d7bff">{sc.fingerprint.method}</Tag>285 <span className="mono text-fg">286 {sc.fingerprint.hostname}287 <span className="text-fg-2">{sc.fingerprint.path_pattern}</span>288 </span>289 {sc.fingerprint.graphql_operation && <Tag color="#ff7ad9">{sc.fingerprint.graphql_operation}</Tag>}290 <span className="ml-auto flex gap-1">291 {sc.schema.candidate_entity_types.slice(0, 4).map((c) => (292 <Tag key={c.type + c.path} color="#38e1ff">293 {c.type} {Number(c.confidence).toFixed(2)}294 </Tag>295 ))}296 </span>297 <span className="mono text-xs text-dim">seen {sc.observed_count}× · {sc.schema.object_count} objects · #{sc.shape_hash.slice(0, 8)}</span>298 </summary>299 <div className="px-3 pb-3">300 <div className="text-[11px] text-dim mono mb-2 break-all">sample: {sc.sample_url}</div>301 <Table head={["field path", "inferred semantic", "conf", "example"]} dense>302 {sc.schema.fields.slice(0, 60).map((f) => (303 <tr key={f.path}>304 <td className="mono text-[11px] text-fg-2 max-w-[520px] truncate">{f.path}</td>305 <td>{f.semantic ? <Tag color="#43e69a">{f.semantic.kind}</Tag> : "—"}</td>306 <td className="mono text-[11px]">{f.semantic ? Number(f.semantic.confidence).toFixed(2) : ""}</td>307 <td className="mono text-[11px] text-dim max-w-[300px] truncate">{f.example}</td>308 </tr>309 ))}310 </Table>311 </div>312 </details>313 ))314 )}315 </div>316 </Panel>317 )}318319 {tab === "media" && (320 <Panel title="Media" sub="videos detected on DOM, network and entity surfaces" pad={false}>321 <div className="p-4 grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-3">322 {media.loading ? (323 <Skeleton rows={6} />324 ) : media.data?.length ? (325 media.data.map((m) => (326 <div key={m.fingerprint} className="rounded-lg border border-line bg-bg-2/40 overflow-hidden">327 {m.frames?.length ? (328 <div className="grid grid-cols-5 gap-px bg-line">329 {m.frames.map((f) => (330 // eslint-disable-next-line @next/next/no-img-element331 <img key={f} src={`/media/${f.split("/media/")[1] ?? ""}`} alt="" className="aspect-video object-cover w-full" />332 ))}333 </div>334 ) : m.thumbnail_url ? (335 // eslint-disable-next-line @next/next/no-img-element336 <img src={m.thumbnail_url} alt="" className="aspect-video object-cover w-full" />337 ) : (338 <div className="aspect-video grid place-items-center text-dim text-xs">no frame</div>339 )}340 <div className="p-3 space-y-1">341 <div className="text-sm truncate">{m.title ?? m.platform_media_id}</div>342 <div className="text-[11px] text-dim mono flex flex-wrap gap-x-3">343 <span>{m.author ?? ""}</span>344 <span>{fmtDuration(m.duration_s)}</span>345 {m.width ? <span>{m.width}×{m.height}</span> : null}346 {m.delivery?.kind && <span>{m.delivery.kind}</span>}347 {m.delivery?.hostnames?.length ? <span>{m.delivery.hostnames.slice(0, 2).join(", ")}</span> : null}348 </div>349 <Link href={`/entities/${encodeURIComponent(m.fingerprint)}`} className="text-[11px] text-cyan">evidence →</Link>350 </div>351 </div>352 ))353 ) : (354 <Empty>no media yet</Empty>355 )}356 </div>357 </Panel>358 )}359360 {tab === "world model" && (361 <Panel title="Social world model" sub="graph of entities and relationships discovered in this session — orange ring = visited by the crawler" pad={false}>362 <div className="p-2">{world.data ? <WorldGraph nodes={world.data.nodes} edges={world.data.edges} height={620} /> : <Skeleton rows={8} />}</div>363 </Panel>364 )}365366 {tab === "events" && (367 <div className="grid grid-cols-1 xl:grid-cols-4 gap-4">368 <Panel title="Event mix">369 <ul className="space-y-1 text-xs mono">370 {(sess?.event_counts ?? [])371 .slice()372 .sort((a, b) => b.n - a.n)373 .map((c) => (374 <li key={c.event_type} className="flex justify-between gap-2">375 <Tag color={eventColor(c.event_type)}>{c.event_type.toLowerCase()}</Tag>376 <span>{c.n}</span>377 </li>378 ))}379 </ul>380 </Panel>381 <Panel title="Raw observation stream" sub={live ? "live" : "session ended — showing the tail"} className="xl:col-span-3" pad={false}>382 <div className="p-2">{live ? <LiveFeed session={id} max={200} height={640} /> : <EventsTail id={id} />}</div>383 </Panel>384 </div>385 )}386 </div>387 );388}389390function EventsTail({ id }: { id: string }) {391 const { data } = useApi<{ event_id: string; event_type: string; step: number; ts: string; payload: Record<string, unknown> }[]>(`/sessions/${id}/events?limit=400`);392 if (!data) return <Skeleton rows={8} />;393 return (394 <ul className="mono text-[11.5px] space-y-0.5 max-h-[640px] overflow-auto scrollbar-thin">395 {data.map((e) => (396 <li key={e.event_id} className="flex gap-2 leading-5 px-1">397 <span className="text-dim">{fmtTime(e.ts)}</span>398 <span className="text-dim w-7 text-right">#{e.step ?? 0}</span>399 <Tag color={eventColor(e.event_type)}>{e.event_type.replace(/_/g, " ").toLowerCase()}</Tag>400 <span className="text-fg-2 truncate">{truncate(JSON.stringify(e.payload), 160)}</span>401 </li>402 ))}403 </ul>404 );405}406